package com.dmcb.trade.business.crawlers.author;

import com.dmcb.common.business.constants.CommonConstant;
import com.dmcb.common.business.services.BaseService;
import com.dmcb.common.business.utils.RegexUtil;
import com.dmcb.common.business.utils.WebUtil;
import com.dmcb.common.web.conversion.JsonResult;
import com.dmcb.common.business.utils.WebClientUtil;
import com.dmcb.trade.business.constants.CrawlConstant;
import com.dmcb.trade.business.entities.Article;
import com.dmcb.trade.business.entities.Author;
import com.dmcb.trade.business.entities.AuthorLink;
import com.dmcb.trade.business.mappers.ArticleCrawlMapper;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;

import java.net.URL;

/**
 * 基础爬取类
 * Created by long on 2016/3/15.
 */
public abstract class BaseCrawler extends BaseService {

    Logger logger = LoggerFactory.getLogger(this.getClass());

    @Autowired
    private ArticleCrawlMapper articleCrawlMapper;
    @Autowired
    protected WebClientUtil webClientUtil;

    /**
     * 定时任务自动爬取更新
     *
     * @param authorId 作者ID
     * @param link     链接
     * @return 消息
     */
    public JsonResult startAuto(int authorId, String link) {
        try {
            // 爬取文章
            HtmlPage htmlPage = webClientUtil.getPageJs(link);
            if (htmlPage == null) {
                return error(CommonConstant.ABNORMAL_MSG);
            }
            Document doc = Jsoup.parse(htmlPage.asXml(), baseUrl(link));
            return crawl(authorId, link, articleCrawlMapper.queryAuthor(authorId), doc, false);
        } catch (Exception e) {
            logger.error("自动更新作者文章出错", e);
            return error(e.getMessage());
        }
    }

    /**
     * 爬取作者文章
     *
     * @param authorId  作者ID
     * @param link      链接
     * @param authorize 是否需要授权 后台不需要 前台需要
     * @param all       是否爬取全部页 true:所有页都需要爬取
     * @return 消息
     */
    public JsonResult startCall(int authorId, String link, boolean authorize, boolean all) {
        // 校验参数
        String msg = check(link);
        if (StringUtils.isNotBlank(msg)) {
            return error(msg);
        }
        // 设置作者
        Author author = articleCrawlMapper.queryAuthor(authorId);
        author = ToutiaoCrawler.getAuthor(authorId, author, articleCrawlMapper);
        try {
            // 正式爬取
            if (authorize) {
                return crawlAuthorize(authorId, link); // 作者提交文章授权链接
            } else {
                return crawlNoAuthorize(authorId, link, author, all);  // 后台直接提交作者列表链接
            }
        } catch (Exception e) {
            logger.error("爬取作者文章出错", e);
            return error(CommonConstant.ABNORMAL_MSG);
        }
    }

    /**
     * 爬取授权文章链接 作者提交 需要校验、确认
     */
    private JsonResult crawlAuthorize(int authorId, String link) throws Exception {
        // 爬取文章
        //如果是头条的文章需要，使用代理
        HtmlPage htmlPage;
        if (RegexUtil.startWith(link, CrawlConstant.TOUTIAO_URL)) {
            WebClientUtil.wc = webClientUtil.initWc(false, false, true);
            webClientUtil.getPage("http://www.toutiao.com/");
            htmlPage = webClientUtil.getPage(link);
        } else {
            htmlPage = webClientUtil.getPageJs(link);
        }
        if (htmlPage == null) {
            return error(CommonConstant.ABNORMAL_MSG);
        }
        Document doc = Jsoup.parse(htmlPage.asXml(), baseUrl(link));
        String body = getBody(doc);
        if (StringUtils.isBlank(body) || !body.contains(CrawlConstant.ARTICLE_AUTHORIZE_WORD)) {
            return error("文章尚未授权，请核对后重新输入");
        }
        // 获取作者链接  有的网站作者标签没有连接
        link = getLink(doc);
        if (StringUtils.isBlank(link)) {
            return error("文章链接有误，请核对后重新输入");
        }
        if (!link.contains("http")) {
            link = url() + link;
        }
        // 爬取作者历史文章
        htmlPage = webClientUtil.getPageJs(link);
        if (htmlPage == null) {
            return error(CommonConstant.ABNORMAL_MSG);
        }
        link = htmlPage.getUrl().toString();
        logger.info("author link = {}", link);
        // 校验链接
        if (articleCrawlMapper.countAuthorLink(link) >= 1) {
            return error("文章链接已绑定");
        }
        // 保存链接并返回
        return success(save(authorId, link));
    }

    public String getLink(Document doc) {
        Element element = doc.select(authorLink()).first();
        if (element == null) {
            return null;
        }
        return element.absUrl("href");
    }

    private AuthorLink save(int authorId, String link) {
        AuthorLink authorLink = new AuthorLink();
        authorLink.setLink(link);
        authorLink.setTag(tag());
        authorLink.setValid(1);
        authorLink.setAuthorId(authorId);
        articleCrawlMapper.insertAuthorLink(authorLink);
        return authorLink;
    }

    /**
     * 后台爬取授权文章链接，直接爬取
     */
    private JsonResult crawlNoAuthorize(int authorId, String link, Author author, boolean all) throws Exception {
        // 爬取作者历史文章
        HtmlPage htmlPage;
        Document doc = null;
        if (RegexUtil.startWith(link, CrawlConstant.TOUTIAO_URL)) {
            //如果是头条文章直接通过POST请求获取文章内容就好
            doc = Jsoup.parse(WebUtil.sendPost(link, ""));
        } else {
            htmlPage = webClientUtil.getPage(link);
            if (htmlPage == null) {
                doc = Jsoup.parse(new URL(link), 6000);
            } else {
                doc = Jsoup.parse(htmlPage.asXml(), baseUrl(link));
            }
            if (htmlPage == null && doc == null) {
                return error(CommonConstant.ABNORMAL_MSG);
            }
            if (!link.startsWith("http://www.toutiao.com") && htmlPage != null) {
                link = htmlPage.getUrl().toString();
            }
            logger.info("author link = {}", link);
        }
        // 保存链接
        AuthorLink authorLink = articleCrawlMapper.queryAuthorLink(link);
        if (authorLink == null) {
            authorLink = save(authorId, link);
        }
        // 更新作者信息
        updateAuthor(doc, author, authorLink);
        // 爬取文章
        JsonResult result = crawl(authorId, link, author, doc, all);
        // 设置链接记录有效
        if (result.isSuccess() && articleCrawlMapper.countInvalidAuthorLink(link) > 0) {
            articleCrawlMapper.setAuthorLinkValid(link);
        }
        return result;
    }

    /**
     * 更新作者信息
     *
     * @param doc    网页
     * @param author 作者
     */
    private void updateAuthor(Document doc, Author author, AuthorLink authorLink) {
        Element element;
        String value;
        // 是否需要更新
        boolean required = false;
        boolean requiredLink = false;
        boolean blank, blankLink;
        // 作者头像
        blank = StringUtils.isBlank(author.getPhotoPath());
        blankLink = StringUtils.isBlank(authorLink.getPhotoPath());
        if (blank || blankLink) {
            element = doc.select(avatarSelect()).first();
            if (element != null) {
                value = element.attr("src");
                if (StringUtils.isNotBlank(value)) {
                    value = value.trim();
                    if (blank) {
                        author.setPhotoPath(value);
                        required = true;
                    }
                    if (blankLink) {
                        authorLink.setPhotoPath(value);
                        requiredLink = true;
                    }
                }
            }
        }
        // 频道名称
        blank = StringUtils.isBlank(author.getChannelName());
        blankLink = StringUtils.isBlank(authorLink.getChannelName());
        if (blank || blankLink) {
            element = doc.select(channelNameSelect()).first();
            if (element != null) {
                value = element.text();
                if (StringUtils.isNotBlank(value)) {
                    value = value.trim();
                    if (blank) {
                        author.setChannelName(value);
                        if (isAuthorName()) {
                            author.setAuthorName(value);
                        }
                        required = true;
                    }
                    if (blankLink) {
                        authorLink.setChannelName(value);
                        requiredLink = true;
                    }
                }
            }
        }
        if (required)
            articleCrawlMapper.updateAuthorInfo(author);
        if (requiredLink)
            articleCrawlMapper.updateAuthorLinkInfo(authorLink);
    }

    /**
     * 校验作者、链接
     *
     * @param link 链接
     * @return 出错信息或空
     */
    protected String check(String link) {
        if (!RegexUtil.startWithDomain(link, url())) {
            return "链接地址错误，请重新输入";
        }
        return "";
    }

    /**
     * 迭代解析、保存文章
     *
     * @return 成功保存数量
     */
    protected JsonResult iterate(int authorId, Document doc, String listSelect, String titleSelect) {
        Elements elements = doc.select(listSelect);
        if (elements.size() == 0) {
            return success(0);
        }
        int succ = 0;
        int exist = 0;
        for (Element element : elements) {
            // 获取标题、链接元素
            Element ele = element.select(titleSelect).first();
            if (ele != null) {
                try {
                    // 校验作者、标题是否已存在
                    Article article = articleCrawlMapper.queryByAuthorTitle(authorId, ele.text().trim());
                    if (article != null) {
                        exist++;
                        if (exist >= 6 && succ == 0) {
                            return success(0);
                        }
                        continue;
                    }
                    String href = ele.attr("href");
                    JsonResult result = parse(authorId, href);
                    succ += result.isSuccess() ? 1 : 0;
                } catch (Exception e) {
                    logger.error("Parser.parse error,{}", ele, e);
                    return error(CommonConstant.ABNORMAL_MSG);
                }
            }
        }
        return success(succ);
    }

    /**
     * 迭代解析、保存文章
     *
     * @return 成功保存数量
     */
    protected JsonResult iterate(int authorId, Document doc) {
        return iterate(authorId, doc, listSelect(), titleSelect());
    }

    /**
     * 返回url主页链接
     */
    protected abstract String url();

    /**
     * 返回url主页链接
     */
    protected String baseUrl(String link) {
        if (url().contains("[") || url().contains("+")) {
            return RegexUtil.getStart(link, url());
        }
        return url();
    }

    /**
     * 返回网站数字标识
     */
    protected abstract int tag();

    /**
     * 返回作者头像选择器
     */
    protected abstract String avatarSelect();


    /**
     * 获取作者链接
     *
     * @return
     */
    protected abstract String authorLink();

    /**
     * 返回作者频道名称选择器
     */
    protected abstract String channelNameSelect();

    /**
     * 频道名称是否同时是作者名称
     *
     * @return 是、否
     */
    protected boolean isAuthorName() {
        return false;
    }

    /**
     * 返回列表选择器
     */
    protected abstract String listSelect();

    /**
     * 返回标题、链接选择器，多选
     */
    protected abstract String titleSelect();


    /**
     * 返回作者文章链接的选择器
     */
    protected abstract String authorSelect();

    /**
     * 爬取文章
     */
    protected abstract JsonResult crawl(int authorId, String link, Author author, Document doc, boolean all);

    /**
     * 爬取、保存链接文章
     *
     * @param authorId 作者ID
     * @param link     链接
     * @return 成功数量 0或1
     */
    protected abstract JsonResult parse(int authorId, String link) throws Exception;

    protected abstract String getBody(Document doc);

}
